# import requests
# from bs4 import BeautifulSoup
#
#
# def scrawler(url, selector):
#
#     headers = {
#         "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
#     }
#     r = requests.get(url, headers=headers)
#     r.encoding='utf-8'
#     codes = r.text
#     # print(codes)
#     bs = BeautifulSoup(codes, 'html.parser')
#     res = bs.select(selector=selector)
#
#     return (item.text for item in res)
#
# url = "http://cjig.ijournals.cn/jig/ch/reader/view_abstract.aspx?file_no=230738&flag=1"
# articlest1=scrawler(url,'div.zynr')
#
#
# for i in articlest1:
#     print(i)
#



import requests
from bs4 import BeautifulSoup


def scrawler(url, selector):

    headers = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
    }
    r = requests.get(url, headers=headers)
    r.encoding='utf-8'
    codes = r.text
    # print(codes)
    bs = BeautifulSoup(codes, 'html.parser')
    res = bs.select(selector=selector)

    return (item.text for item in res)

url = "http://jst.tsinghuajournals.com/article/2024/4358/20240901.htm"
abstract=scrawler(url,'div.abs')
article_body=scrawler(url,'div.article_body')


for i in abstract:
    print(i)
for i in article_body:
    print(i)
